{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6218db4b-a625-4015-813a-3f36875a1074",
   "metadata": {},
   "source": [
    "# 本脚本用于部署CSDC gptq int4 量化过的Baichuan2-13b-chat \n",
    "https://huggingface.co/csdc-atl/Baichuan2-13B-Chat-GPTQ-Int4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c261e5f4-17a8-40da-beb9-599f1717e0fe",
   "metadata": {},
   "source": [
    "### 1. 安装HuggingFace 并下载模型到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "02785614-9268-41c8-85a5-d579490edbbf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sagemaker in /opt/conda/lib/python3.7/site-packages (2.183.0)\n",
      "Requirement already satisfied: attrs<24,>=23.1.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (23.1.0)\n",
      "Requirement already satisfied: boto3<2.0,>=1.26.131 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.28.44)\n",
      "Requirement already satisfied: cloudpickle==2.2.1 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (2.2.1)\n",
      "Requirement already satisfied: google-pasta in /opt/conda/lib/python3.7/site-packages (from sagemaker) (0.2.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.9.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.21.6)\n",
      "Requirement already satisfied: protobuf<5.0,>=3.12 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (4.24.0)\n",
      "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.0.1)\n",
      "Requirement already satisfied: importlib-metadata<7.0,>=1.4.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (6.7.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (23.1)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.3.5)\n",
      "Requirement already satisfied: pathos in /opt/conda/lib/python3.7/site-packages (from sagemaker) (0.3.1)\n",
      "Requirement already satisfied: schema in /opt/conda/lib/python3.7/site-packages (from sagemaker) (0.7.5)\n",
      "Requirement already satisfied: PyYAML~=6.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (6.0.1)\n",
      "Requirement already satisfied: jsonschema in /opt/conda/lib/python3.7/site-packages (from sagemaker) (3.2.0)\n",
      "Requirement already satisfied: platformdirs in /opt/conda/lib/python3.7/site-packages (from sagemaker) (3.10.0)\n",
      "Requirement already satisfied: tblib==1.7.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.7.0)\n",
      "Requirement already satisfied: botocore<1.32.0,>=1.31.44 in /opt/conda/lib/python3.7/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.31.44)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/conda/lib/python3.7/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.0.1)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (0.6.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata<7.0,>=1.4.0->sagemaker) (2.2.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata<7.0,>=1.4.0->sagemaker) (4.7.1)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from google-pasta->sagemaker) (1.14.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema->sagemaker) (0.15.7)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from jsonschema->sagemaker) (65.5.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.7/site-packages (from pandas->sagemaker) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.7/site-packages (from pandas->sagemaker) (2019.3)\n",
      "Requirement already satisfied: ppft>=1.7.6.7 in /opt/conda/lib/python3.7/site-packages (from pathos->sagemaker) (1.7.6.7)\n",
      "Requirement already satisfied: dill>=0.3.7 in /opt/conda/lib/python3.7/site-packages (from pathos->sagemaker) (0.3.7)\n",
      "Requirement already satisfied: pox>=0.3.3 in /opt/conda/lib/python3.7/site-packages (from pathos->sagemaker) (0.3.3)\n",
      "Requirement already satisfied: multiprocess>=0.70.15 in /opt/conda/lib/python3.7/site-packages (from pathos->sagemaker) (0.70.15)\n",
      "Requirement already satisfied: contextlib2>=0.5.5 in /opt/conda/lib/python3.7/site-packages (from schema->sagemaker) (0.6.0.post1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /opt/conda/lib/python3.7/site-packages (from botocore<1.32.0,>=1.31.44->boto3<2.0,>=1.26.131->sagemaker) (1.26.16)\n",
      "\u001b[33mDEPRECATION: pyodbc 4.0.0-unsupported has a non-standard version number. pip 23.3 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pyodbc or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install huggingface-hub -Uqq \n",
    "!pip install -U sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9e6bd7ee-16a3-4f5a-8857-8bbba83eb9e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "local_model_path = Path(\"./LLM_bc2_13b_int4_model\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c3469632-4174-4df4-a7d1-ef167561c626",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "local_model_path.mkdir(exist_ok=True)\n",
    "model_name = \"csdc-atl/Baichuan2-13B-Chat-Int4\"\n",
    "# commit_hash = \"670d17ee403f45334f53121d72feff623cc37de1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "94e8abc5-a58e-40e2-b1e6-fbf48307c716",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c889bc21afba4874b3bf4e70e89b1993",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Fetching 17 files', max=17.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "746fc33889b94d9790c822e0fe7d0f58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)aichuan2%20Model.pdf', max=202809.0, style…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1c9ec5614d14b08acb64b479af90d71",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)5%8D%8F%E8%AE%AE.pdf', max=257546.0, style…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34cdb9a03236441597b4fb1dd0f01765",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)2bbb8087/config.json', max=1017.0, style=P…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5465b0a5495f4a6bbd63f6b54ea1708b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)b42bbb8087/README.md', max=17278.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3f3927f9af44d84b38d0ebe302a10a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)b8087/.gitattributes', max=1519.0, style=P…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5ae398b1c4e452ea41b57528fa4cd57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)guration_baichuan.py', max=1560.0, style=P…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07e4f379949b42c182aab316ce827e9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)/generation_utils.py', max=2966.0, style=P…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99188276acb64b32abd29ad96af24b22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)neration_config.json', max=285.0, style=Pr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2462a45cd00498a92a6d881f5dc5df0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)42bbb8087/handler.py', max=1099.0, style=P…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6d51cb22fdb40d5916465e56055b2f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)modeling_baichuan.py', max=29466.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96857163a04c4236947be5ba521464ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)bbb8087/quantizer.py', max=9112.0, style=P…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "647beb8eba4347a98915a8f8903b792b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)nization_baichuan.py', max=9030.0, style=P…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b3739a147f5435e9698b362583b9110",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)quantize_config.json', max=212.0, style=Pr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60aadb64a8684b0aa591f5dd39c7668f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)cial_tokens_map.json', max=544.0, style=Pr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ac718582021401f9e53f6c658c5f35d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading (…)okenizer_config.json', max=954.0, style=Pr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "610d8dbe2732469fac8375807b7a47a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading model.safetensors', max=9135747280.0, style=P…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93e363a4931a4779b48e0da82e491d51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading tokenizer.model', max=2001107.0, style=Progre…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snapshot_download(repo_id=model_name, \n",
    "                  # revision=commit_hash,\n",
    "                  cache_dir=local_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d666c79-b039-4258-ac3b-46b19e63c3b8",
   "metadata": {},
   "source": [
    "### 2. 把模型拷贝到S3为后续部署做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e9431deb-6359-442d-847b-1563f8dd3854",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/boto3/compat.py:82: PythonDeprecationWarning: Boto3 will no longer support Python 3.7 starting December 13, 2023. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.8 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/\n",
      "  warnings.warn(warning, PythonDeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import sagemaker\n",
    "from sagemaker import image_uris\n",
    "import boto3\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "\n",
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "\n",
    "region = sess._region_name\n",
    "account_id = sess.account_id()\n",
    "\n",
    "s3_client = boto3.client(\"s3\")\n",
    "sm_client = boto3.client(\"sagemaker\")\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "40dd8f16-ae7c-48bf-8e52-1a15425fa74d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3_code_prefix: LLM-RAG/workshop/LLM_bc2_13b_int4_deploy_code\n",
      "model_snapshot_path: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087\n"
     ]
    }
   ],
   "source": [
    "s3_model_prefix = \"LLM-RAG/workshop/LLM_bc2_13b_int4_model\"  # folder where model checkpoint will go\n",
    "model_snapshot_path = list(local_model_path.glob(\"**/snapshots/*\"))[0]\n",
    "s3_code_prefix = \"LLM-RAG/workshop/LLM_bc2_13b_int4_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "067292c9-c066-4649-a61f-b460a24da584",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/README.md to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/README.md\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/.gitattributes to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/.gitattributes\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/config.json\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/handler.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/handler.py\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/generation_utils.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/generation_utils.py\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/generation_config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/generation_config.json\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/quantize_config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/quantize_config.json\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/special_tokens_map.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/special_tokens_map.json\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/modeling_baichuan.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/modeling_baichuan.py\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/quantizer.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/quantizer.py\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/tokenizer_config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/tokenizer_config.json\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/configuration_baichuan.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/configuration_baichuan.py\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/Baichuan2 模型社区许可协议.pdf to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/Baichuan2 模型社区许可协议.pdf\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/Community License for Baichuan2 Model.pdf to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/Community License for Baichuan2 Model.pdf\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/tokenization_baichuan.py to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/tokenization_baichuan.py\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/tokenizer.model to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/tokenizer.model\n",
      "upload: LLM_bc2_13b_int4_model/models--csdc-atl--Baichuan2-13B-Chat-Int4/snapshots/d9e759654b6cf2999bfd5d2b58e461b42bbb8087/model.safetensors to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/model.safetensors\n"
     ]
    }
   ],
   "source": [
    "!aws s3 cp --recursive {model_snapshot_path} s3://{bucket}/{s3_model_prefix}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "696b70c3-90f1-4175-95bf-568bafbcd383",
   "metadata": {},
   "source": [
    "### 3. 模型部署准备（entrypoint脚本，容器镜像，服务配置）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6f7c4277-4480-42c6-aee6-1fbcca94eb82",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.23.0-deepspeed0.9.5-cu118\n"
     ]
    }
   ],
   "source": [
    "# inference_image_uri = (\n",
    "#     f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\"\n",
    "# )\n",
    "\n",
    "inference_image_uri = image_uris.retrieve(\n",
    "    framework=\"djl-deepspeed\",\n",
    "    region=sess.boto_session.region_name,\n",
    "    version=\"0.23.0\"\n",
    ")\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8d771bdb-11d2-45d2-9bef-face29221838",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mkdir -p LLM_bc2_13b_int4_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e5348ecb-43df-4094-97d8-a6723004862a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_bc2_13b_int4_deploy_code/model.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_bc2_13b_int4_deploy_code/model.py\n",
    "from djl_python import Input, Output\n",
    "import torch\n",
    "import logging\n",
    "import math\n",
    "import os\n",
    "\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer,set_seed\n",
    "from transformers.generation import GenerationConfig\n",
    "STOP_flag = \"[DONE]\"\n",
    "\n",
    "\n",
    "def load_model(properties):\n",
    "    tensor_parallel = properties[\"tensor_parallel_degree\"]\n",
    "    model_location = properties['model_dir']\n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_location, use_fast=True, trust_remote_code=True)\n",
    "    model = AutoModelForCausalLM.from_pretrained(model_location, device_map=\"auto\", torch_dtype=torch.float16, trust_remote_code=True)\n",
    "    model.generation_config = GenerationConfig.from_pretrained(model_location)\n",
    "\n",
    "    return model, tokenizer\n",
    "\n",
    "\n",
    "model = None\n",
    "tokenizer = None\n",
    "generator = None\n",
    "\n",
    "\n",
    "def construct_message(history,prompt):\n",
    "    message = []\n",
    "    for question, answer in history:\n",
    "        message.append({\"role\":\"user\",\"content\":question})\n",
    "        message.append({\"role\":\"assistant\",\"content\":answer})\n",
    "    message.append({\"role\":\"user\",\"content\":prompt})\n",
    "    return message\n",
    "\n",
    "def stream_items(prompt, history, max_length, top_p, temperature):\n",
    "    global model, tokenizer\n",
    "    size = 0\n",
    "    response = \"\"\n",
    "    model.generation_config.max_new_tokens = max_length\n",
    "    model.generation_config.top_p = top_p if top_p < 1 else 0.95\n",
    "    \n",
    "    messages = construct_message(history,prompt)\n",
    "    res_generator = model.chat(tokenizer, messages,stream=True)\n",
    "    for response in res_generator:\n",
    "        this_response = response[size:]\n",
    "        size = len(response)\n",
    "        stream_buffer = { \"outputs\":this_response,\"finished\": False}\n",
    "        yield stream_buffer\n",
    "    ## stop\n",
    "    # yield {\"query\": prompt, \"outputs\": STOP_flag, \"response\": response, \"history\": [], \"finished\": True}\n",
    "    \n",
    "\n",
    "\n",
    "def handle(inputs: Input):\n",
    "    # set_seed(1)\n",
    "    global model, tokenizer\n",
    "    if not model:\n",
    "        model, tokenizer = load_model(inputs.get_properties())\n",
    "\n",
    "    if inputs.is_empty():\n",
    "        return None\n",
    "    data = inputs.get_as_json()\n",
    "    \n",
    "    input_sentences = data[\"inputs\"]\n",
    "    params = data[\"parameters\"]\n",
    "    history = data[\"history\"]\n",
    "    stream = data.get('stream')\n",
    "    print(f'input prompt:{input_sentences}')    \n",
    "    outputs = Output()\n",
    "    if stream:\n",
    "        outputs.add_property(\"content-type\", \"application/jsonlines\")\n",
    "        outputs.add_stream_content(stream_items(input_sentences,history=history,**params))\n",
    "    else:\n",
    "        messages = construct_message(history,input_sentences)\n",
    "        response = model.chat(tokenizer, messages,stream=False)\n",
    "        result = {\"outputs\": response, \"history\" : []}\n",
    "        outputs.add_as_json(result)\n",
    "        \n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a06d1e60-3914-4059-a08f-05ac26761165",
   "metadata": {},
   "source": [
    "#### Note: option.s3url 需要按照自己的账号进行修改"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8996fe44-8e70-468b-abc1-38187cb33f4f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_bc2_13b_int4_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_bc2_13b_int4_deploy_code/serving.properties\n",
    "engine=Python\n",
    "option.tensor_parallel_degree=1\n",
    "option.enable_streaming=True\n",
    "option.predict_timeout=240\n",
    "option.s3url = s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_model/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feef22a2-27b9-4018-a46b-6a99b532512f",
   "metadata": {},
   "source": [
    "#### 注意: 必须把transformers升级到4.27.1以上，否则会出现 [Issue344](https://github.com/THUDM/ChatGLM-6B/issues/344)\n",
    "\n",
    "如果是中国区建议添加国内的pip镜像,如下代码所示\n",
    "```\n",
    "%%writefile LLM_chatglm_deploy_code/requirements.txt\n",
    "-i https://pypi.tuna.tsinghua.edu.cn/simple\n",
    "transformers==4.28.1\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7b7e76c6-6dbc-47fc-9f47-4765c526ab76",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_bc2_13b_int4_deploy_code/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_bc2_13b_int4_deploy_code/requirements.txt\n",
    "transformers==4.32.0\n",
    "accelerate\n",
    "colorama\n",
    "bitsandbytes\n",
    "sentencepiece\n",
    "transformers_stream_generator\n",
    "cpm_kernels\n",
    "xformers\n",
    "auto-gptq\n",
    "optimum>=1.12.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0ae6734a-aacd-410d-818d-0a962697c3c4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LLM_bc2_13b_int4_deploy_code/\n",
      "LLM_bc2_13b_int4_deploy_code/model.py\n",
      "LLM_bc2_13b_int4_deploy_code/requirements.txt\n",
      "LLM_bc2_13b_int4_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "!rm model.tar.gz\n",
    "!cd LLM_bc2_13b_int4_deploy_code && rm -rf \".ipynb_checkpoints\"\n",
    "!tar czvf model.tar.gz LLM_bc2_13b_int4_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0f77dc76-6d8c-4665-ba88-f03e887c136c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S3 Code or Model tar ball uploaded to --- > s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/LLM_bc2_13b_int4_deploy_code/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "s3_code_artifact = sess.upload_data(\"model.tar.gz\", bucket, s3_code_prefix)\n",
    "print(f\"S3 Code or Model tar ball uploaded to --- > {s3_code_artifact}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5853daa-b8a3-4485-8c0a-64bf83e93a18",
   "metadata": {},
   "source": [
    "### 4. 创建模型 & 创建endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ef974ca1-9638-45a8-9145-ea9d03b2b072",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bc2-13b-gptq-2023-09-20-05-04-44-477\n",
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.23.0-deepspeed0.9.5-cu118\n",
      "Created Model: arn:aws:sagemaker:us-east-2:946277762357:model/bc2-13b-gptq-2023-09-20-05-04-44-477\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "import boto3\n",
    "\n",
    "model_name = name_from_base(f\"bc2-13b-gptq\") #Note: Need to specify model_name\n",
    "print(model_name)\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")\n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": s3_code_artifact\n",
    "    },\n",
    "    \n",
    ")\n",
    "\n",
    "\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "\n",
    "print(f\"Created Model: {model_arn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "233bb3a4-d737-41ad-8fcc-7082c6278e8c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EndpointConfigArn': 'arn:aws:sagemaker:us-east-2:946277762357:endpoint-config/bc2-13b-gptq-2023-09-20-05-04-44-477-config',\n",
       " 'ResponseMetadata': {'RequestId': '38cdcf5c-f1c5-4e17-8a22-1296b2fb74e0',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '38cdcf5c-f1c5-4e17-8a22-1296b2fb74e0',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '124',\n",
       "   'date': 'Wed, 20 Sep 2023 05:04:52 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "#Note: ml.g4dn.2xlarge 也可以选择\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.g5.2xlarge\",\n",
    "            \"InitialInstanceCount\": 1,\n",
    "            # \"VolumeSizeInGB\" : 400,\n",
    "            # \"ModelDataDownloadTimeoutInSeconds\": 2400,\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 15*60,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "endpoint_config_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "734a39b0-473e-4421-94c8-74d2b4105038",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created Endpoint: arn:aws:sagemaker:us-east-2:946277762357:endpoint/bc2-13b-gptq-2023-09-20-05-04-44-477-endpoint\n"
     ]
    }
   ],
   "source": [
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(f\"Created Endpoint: {create_endpoint_response['EndpointArn']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1262e826-a810-401d-a5a9-f62febb24e5f",
   "metadata": {},
   "source": [
    "#### 持续检测模型部署进度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "08969928-6b9e-4d9c-a033-a31f5f77bdfb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: InService\n",
      "Arn: arn:aws:sagemaker:us-east-2:946277762357:endpoint/bc2-13b-gptq-2023-09-20-05-04-44-477-endpoint\n",
      "Status: InService\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp[\"EndpointStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp[\"EndpointStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d985b427-3959-46f7-9a50-5a2b45e2d513",
   "metadata": {},
   "source": [
    "### 5. 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fd6d7d0d-4160-49a9-91be-a3e2a4e98ac7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: boto3 in /opt/conda/lib/python3.7/site-packages (1.28.47)\n",
      "Collecting boto3\n",
      "  Obtaining dependency information for boto3 from https://files.pythonhosted.org/packages/fe/19/6699107cf1cad26b5ba4aa6206037151e30ab5673f0c5041de6ddfacbbe5/boto3-1.28.51-py3-none-any.whl.metadata\n",
      "  Downloading boto3-1.28.51-py3-none-any.whl.metadata (6.7 kB)\n",
      "Requirement already satisfied: botocore in /opt/conda/lib/python3.7/site-packages (1.31.47)\n",
      "Collecting botocore\n",
      "  Obtaining dependency information for botocore from https://files.pythonhosted.org/packages/17/33/041d4ec33230b68c077ac11c92cbfc71cbca685527c1fb6a0067fbcaab63/botocore-1.31.51-py3-none-any.whl.metadata\n",
      "  Downloading botocore-1.31.51-py3-none-any.whl.metadata (6.0 kB)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/conda/lib/python3.7/site-packages (from boto3) (1.0.1)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from boto3) (0.6.1)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/conda/lib/python3.7/site-packages (from botocore) (2.8.2)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /opt/conda/lib/python3.7/site-packages (from botocore) (1.26.16)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil<3.0.0,>=2.1->botocore) (1.14.0)\n",
      "Downloading boto3-1.28.51-py3-none-any.whl (135 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m135.8/135.8 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading botocore-1.31.51-py3-none-any.whl (11.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.2/11.2 MB\u001b[0m \u001b[31m22.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25h\u001b[33mDEPRECATION: pyodbc 4.0.0-unsupported has a non-standard version number. pip 23.3 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pyodbc or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0mInstalling collected packages: botocore, boto3\n",
      "  Attempting uninstall: botocore\n",
      "    Found existing installation: botocore 1.31.47\n",
      "    Uninstalling botocore-1.31.47:\n",
      "      Successfully uninstalled botocore-1.31.47\n",
      "  Attempting uninstall: boto3\n",
      "    Found existing installation: boto3 1.28.47\n",
      "    Uninstalling boto3-1.28.47:\n",
      "      Successfully uninstalled boto3-1.28.47\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "aiobotocore 2.4.2 requires botocore<1.27.60,>=1.27.59, but you have botocore 1.31.51 which is incompatible.\n",
      "awscli 1.29.26 requires botocore==1.31.26, but you have botocore 1.31.51 which is incompatible.\n",
      "awscli 1.29.26 requires rsa<4.8,>=3.1.2, but you have rsa 4.9 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed boto3-1.28.51 botocore-1.31.51\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install -U boto3 botocore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "e56bfdaa-3469-4784-aa8a-e32177cde3f2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.87 ms, sys: 675 µs, total: 5.54 ms\n",
      "Wall time: 5.78 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import json\n",
    "import boto3\n",
    "\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")\n",
    "\n",
    "parameters = {\n",
    "  \"max_length\": 1024,\n",
    "  \"temperature\": 0.5,\n",
    "  \"top_p\":0.9\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ae5983aa-0c91-4c78-a63f-7192a39a8cfb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import io\n",
    "\n",
    "\n",
    "class StreamScanner:\n",
    "    \"\"\"\n",
    "    A helper class for parsing the InvokeEndpointWithResponseStream event stream. \n",
    "    \n",
    "    The output of the model will be in the following format:\n",
    "    ```\n",
    "    b'{\"outputs\": [\" a\"]}\\n'\n",
    "    b'{\"outputs\": [\" challenging\"]}\\n'\n",
    "    b'{\"outputs\": [\" problem\"]}\\n'\n",
    "    ...\n",
    "    ```\n",
    "    \n",
    "    While usually each PayloadPart event from the event stream will contain a byte array \n",
    "    with a full json, this is not guaranteed and some of the json objects may be split across\n",
    "    PayloadPart events. For example:\n",
    "    ```\n",
    "    {'PayloadPart': {'Bytes': b'{\"outputs\": '}}\n",
    "    {'PayloadPart': {'Bytes': b'[\" problem\"]}\\n'}}\n",
    "    ```\n",
    "    \n",
    "    This class accounts for this by concatenating bytes written via the 'write' function\n",
    "    and then exposing a method which will return lines (ending with a '\\n' character) within\n",
    "    the buffer via the 'readlines' function. It maintains the position of the last read \n",
    "    position to ensure that previous bytes are not exposed again. \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.buff = io.BytesIO()\n",
    "        self.read_pos = 0\n",
    "        \n",
    "    def write(self, content):\n",
    "        self.buff.seek(0, io.SEEK_END)\n",
    "        self.buff.write(content)\n",
    "        \n",
    "    def readlines(self):\n",
    "        self.buff.seek(self.read_pos)\n",
    "        for line in self.buff.readlines():\n",
    "            if line[-1] != b'\\n':\n",
    "                self.read_pos += len(line)\n",
    "                yield line[:-1]\n",
    "                \n",
    "    def reset(self):\n",
    "        self.read_pos = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "591830c2-cc7f-49b3-9293-ed8f1ec81093",
   "metadata": {},
   "source": [
    "## Stream "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "84f9219f-fa4d-413e-b02d-2047142b4a79",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "床前明月光是唐代诗人李白创作的一首脍炙人口的诗《静夜思》中的第一句。time:3.5592293739318848 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "\n",
    "# prompts1 = \"\"\"what's the weather like there?\"\"\"\n",
    "# history = [\n",
    "#         [\n",
    "#             \"what is the capital city of china?\",\n",
    "#             \"Beijing.\"\n",
    "#         ]\n",
    "#     ]\n",
    "# prompts1 = \"\"\"AWS Clean Rooms 的FAQ文档有提到 Q: 是否发起者和数据贡献者都会被收费？A: 是单方收费，只有查询的接收方会收费。\n",
    "# 请问AWS Clean Rooms是多方都会收费吗？\n",
    "# \"\"\"\n",
    "# prompts1 = \"\"\"写一篇500字的科幻小说，背景关于宇宙战争\"\"\"\n",
    "prompts1 = \"\"\"床前明月光是谁写的\"\"\"\n",
    "history = []\n",
    "# endpoint_name = 'bc2-13b-stream-2023-09-07-10-59-15-557-endpoint'\n",
    "\n",
    "start = time.time()\n",
    "response_model = smr_client.invoke_endpoint_with_response_stream(\n",
    "            EndpointName=endpoint_name,\n",
    "            Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": prompts1,\n",
    "                \"parameters\": parameters,\n",
    "                \"history\" : history,\n",
    "                \"stream\":True\n",
    "            }\n",
    "            ),\n",
    "            ContentType=\"application/json\",\n",
    "        )\n",
    "\n",
    "event_stream = response_model['Body']\n",
    "scanner = StreamScanner()\n",
    "for event in event_stream:\n",
    "    scanner.write(event['PayloadPart']['Bytes'])\n",
    "    for line in scanner.readlines():\n",
    "        try:\n",
    "            resp = json.loads(line)\n",
    "            # print(resp)\n",
    "            print(resp.get(\"outputs\")['outputs'], end='')\n",
    "        except Exception as e:\n",
    "            # print(line)\n",
    "            continue\n",
    "print (f\"time:{time.time()-start} s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "589a78be-0ea5-4e59-b262-07eb15290776",
   "metadata": {},
   "source": [
    "## None Stream"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "4ce523a2-fb85-4eac-bd17-12789073a92d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time:15.329690217971802 s\n",
      "在遥远的未来，宇宙中的星际文明已经发展到了前所未有的高度。在这个时代，科技的力量已经超越了人类的想象，星际战争成为了一种常态。在这个宇宙舞台上，两个强大的星际帝国——赛博特帝国和诺瓦帝国，为了争夺资源、领土和霸权，展开了一场空前绝后的宇宙战争。\n",
      "\n",
      "赛博特帝国的皇帝，赛博特五世，是一个充满野心和智慧的统治者。他带领着赛博特人民，凭借着先进的科技和强大的军队，征服了一个又一个星球，不断扩大帝国的疆域。而诺瓦帝国，虽然实力同样强大，但在皇帝的统治下，却显得有些保守和犹豫。\n",
      "\n",
      "在这场宇宙战争中，双方都投入了大量的资源和兵力。赛博特帝国的舰队如同一道钢铁洪流，席卷了整个宇宙；而诺瓦帝国的舰队，则如同一群凶猛的猎豹，紧紧地跟在赛博特帝国的身后，伺机发动攻击。\n",
      "\n",
      "在这场战争中，有一个名叫艾瑞克的年轻士兵。他是赛博特帝国的一名精英战士，拥有着卓越的战斗技巧和坚定的信念。他的家人和朋友都在一场突如其来的战争中丧生，这让他对诺瓦帝国充满了仇恨。为了复仇，他义无反顾地加入了赛博特帝国的军队，决心为帝国献出自己的一切。\n",
      "\n",
      "然而，在一场激战中，艾瑞克被诺瓦帝国的士兵俘虏了。面对敌人的威胁和羞辱，艾瑞克并没有放弃。他利用自己的智慧和勇气，成功地逃脱了诺瓦帝国的监狱，重新回到了赛博特帝国的军队中。\n",
      "\n",
      "在这场宇宙战争的后期，赛博特帝国和诺瓦帝国之间的战斗变得越来越激烈。双方的舰队在星空中展开了惊心动魄的厮杀，无数星球在战火中化为灰烬。而艾瑞克，也在一次次战斗中，逐渐成长为了一名英勇无畏的战士。\n",
      "\n",
      "然而，在战争的最后关头，一个意想不到的事件发生了。\n"
     ]
    }
   ],
   "source": [
    "prompts1 = \"\"\"AWS Clean Rooms 的FAQ文档有提到 Q: 是否发起者和数据贡献者都会被收费？A: 是单方收费，只有查询的接收方会收费。\n",
    "请问AWS Clean Rooms是多方都会收费吗？\n",
    "\"\"\"\n",
    "prompts1 = \"\"\"写一篇500字的科幻小说，背景关于宇宙战争\"\"\"\n",
    "# prompts1 = \"\"\"床前明月光是谁写的\"\"\"\n",
    "# prompts1 = \"\"\"那里有什么景点\"\"\"\n",
    "# history = [\n",
    "#         [\n",
    "#             \"中国的首都是哪里\",\n",
    "#             \"北京.\"\n",
    "#         ]\n",
    "#     ]\n",
    "start = time.time()\n",
    "response_model = smr_client.invoke_endpoint(\n",
    "            EndpointName=endpoint_name,\n",
    "            Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": prompts1,\n",
    "                \"parameters\": parameters,\n",
    "                \"history\" : [],\n",
    "            }\n",
    "            ),\n",
    "            ContentType=\"application/json\",\n",
    "        )\n",
    "\n",
    "resp = response_model['Body'].read()\n",
    "end = time.time()-start\n",
    "print (f\"\\ntime:{end} s\")\n",
    "# print(resp.decode('utf8'))\n",
    "print(json.loads(resp)['outputs'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d670595a-d1a5-43de-9801-86482a07e270",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def construct_message(history,prompt):\n",
    "    message = []\n",
    "    for question, answer in history:\n",
    "        message.append({\"role\":\"user\",\"content\":question})\n",
    "        message.append({\"role\":\"assistant\",\"content\":answer})\n",
    "    message.append({\"role\":\"user\",\"content\":prompt})\n",
    "    return message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "d577e076-52b2-4257-a447-1d3a5813d7ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts1 = \"\"\"那里有什么景点\"\"\"\n",
    "history = [\n",
    "        [\n",
    "            \"中国的首都是哪里\",\n",
    "            \"北京.\"\n",
    "        ]\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "b6f3eac4-b633-4d8c-882f-7003eeb74dd1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "messages=construct_message(history,prompts1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "efd4ca1b-534e-43cc-ae39-65629007d4f4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def _parse_messages(messages, split_role=\"user\"):\n",
    "    system, rounds = \"\", []\n",
    "    round = []\n",
    "    for i, message in enumerate(messages):\n",
    "        if message[\"role\"] == \"system\":\n",
    "            assert i == 0\n",
    "            system = message[\"content\"]\n",
    "            continue\n",
    "        if message[\"role\"] == split_role and round:\n",
    "            rounds.append(round)\n",
    "            round = []\n",
    "        round.append(message)\n",
    "    if round:\n",
    "        rounds.append(round)\n",
    "    return system, rounds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "e2c07d5a-2098-4463-b887-cd60d91484f7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, '中', '国', '的', '首', '都', '是', '哪', '里', 2, '北', '京', '.', 1, '那', '里', '有', '什', '么', '景', '点']\n"
     ]
    }
   ],
   "source": [
    "system, rounds = _parse_messages(messages)\n",
    "history_tokens = []\n",
    "for round in rounds[::-1]:\n",
    "    round_tokens = []\n",
    "    \n",
    "    for message in round:\n",
    "        if message[\"role\"] == \"user\":\n",
    "            round_tokens.append(1)\n",
    "        else:\n",
    "            round_tokens.append(2)\n",
    "        round_tokens.extend(message[\"content\"])\n",
    "    history_tokens = round_tokens + history_tokens  # concat left\n",
    "print(history_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "5d45102e-e47e-402b-ba71-856921da3fcc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, '中', '国', '的', '首', '都', '是', '哪', '里', 2, '北', '京', '.']"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "5c370da7-193f-438a-8ddd-a9c9420af18e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[{'role': 'user', 'content': '那里有什么景点'}],\n",
       " [{'role': 'user', 'content': '中国的首都是哪里'},\n",
       "  {'role': 'assistant', 'content': '北京.'}]]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rounds[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "317c075f-f51d-4c87-87c5-09f2f5f5d9af",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[{'role': 'user', 'content': '中国的首都是哪里'},\n",
       "  {'role': 'assistant', 'content': '北京.'}],\n",
       " [{'role': 'user', 'content': '那里有什么景点'}]]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rounds"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21c8b703-e312-4964-8be9-a754468e07cd",
   "metadata": {},
   "source": [
    "#### 清除模型Endpoint和config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f70d116f-4fb1-4f04-8732-3d6e4fb520de",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint --endpoint-name {endpoint_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "184e4d1d-3d62-43df-9b17-5d64ece928bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint-config --endpoint-config-name {endpoint_config_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "707e8f09",
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws sagemaker delete-model --model-name {model_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2890ad5-2db4-4410-be1a-5f71322f57ea",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-2:429704687514:image/datascience-1.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
